[root@k8s-m1 txt_dir]# cat txt
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-10DF/NGS180312-10DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-10DF/NGS180312-10DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-11DF/NGS180312-11DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-11DF/NGS180312-11DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-7DF/NGS180312-7DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-7DF/NGS180312-7DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF/NGS180312-9DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF/NGS180312-9DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180315.tar.gz
./180315_C70121_0005_000000000-BLN2G/fastq/NGS180312-8DJ_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF/NGS180312-9DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180315.tar.gz
./180315_C70121_0005_000000000-BLN2G/fastq/NGS180312-8DJ_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-9DF_R2.fastq.gz

最后一个域里去掉重复次数大于1的行

[root@k8s-m1 txt_dir]# awk -F / '{a[$NF]++;b[$NF]=$0;if(a[$NF]>1)delete b[$NF]}END{for(i in a){if(b[i])print b[i]}}' txt
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-11DF/NGS180312-11DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-7DF/NGS180312-7DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-7DF/NGS180312-7DF_R1.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-10DF/NGS180312-10DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-11DF/NGS180312-11DF_R2.fastq.gz
./180315_C70121_0005_000000000-BLN2G/42geneLung180315/NGS180312-10DF/NGS180312-10DF_R1.fastq.gz
